#!/usr/bin/python -S
"""JSON Pattern -- Minilanguage for extracting structured data from text.


SYNTAX

  "Instructions" are enclosed in the metacharacters { and }.

  EXCEPT: Anything of the format {n} or {m,n} is treated as a "literal", so the
  underlying regex engine can interpret it.

  The initial character (after '{' ) specifies what kind of instruction it is:

  - Instructions starting with : are user-defined subpatterns.
  - Instructions starting with . are literal constants, e.g. {.space},
    {.meta-left}
  - Any non-alphanumeric character except . or : is a regex delimiter (see Named
    Capture below)

  Strings *outside* metacharacters are *regular expressions*, and are
  interpreted by the host language regex engine.
  
  The program is compiled into a tree structure:
  
  - A block is a sequence of fragments.
  - A fragment may be a string literal, regex literal, or a block.


Blocks:

  {repeated type $name | filter} - Repetition ('type' is  *  +  3  3,6  etc.)
  {value $<name> | filter} - Start a new named section
  {optional $<name> | filter} - Like ?
  {choices $<name> | filter} {/or} - Alternation, like (ab|bc|cd)

  $ means "scalar capture".  The value of the block is everything matched,
  rather than the values of the sub-blocks.

  The filter is applied to the captured value before it's included in the JSON
  output.


Named Capture:

  { regex name } - shorthand for {value $name}.  Notice that the $ is not
  necessary in the shorthand.
  
  { regex name | filter} - Named capture with a filter

    * The first character is the regex delimiter, and should be a space by
      default.
    * If a space is present in the pattern, then use / as the delimiter:
      {/[^ ]/ non-space}
    * (If you have both / and space, then choose some other punctuation
      that doesn't appear in the regex as a delimiter)
    * { and } are special metacharacters, so the regex can't contain them.  If
      you need them, use the {/value $name} form.
    * { and } CAN be used as literals, but not regular expression
      metacharacters.

Directives:
  
  Directives change how the program text is interpreted.

  Normally, the rules are:
  
  1) Whitespace is stripped at the beginning and end of lines, so you can use
  indentation to indicate structure in your pattern, and so you can wrap it
  across multiple lines.

  2) Whitespace between instructions and regex patterns is stripped.

  A few directives change this:

  {!literal}
    .*
  {!end} 
  
  Literals cause regex metacharacters not to be interpreted.
    TODO: Measure performance here.  Regex matching is in C, and might
    outperform Python string comparison.

  {!raw-space}
    Foo {blah} bar
  {!end} 
  Raw space means that whitespace between instructions and regexes should be
  treated as literal, rather than stripped like they usually are.

Literals:

  {.meta-left} - same as \{  (literal {, not regex)
  {.meta-right} - same as \}  (literal }, not regex)

    \} and \{ should suffice everywhere.  These are used for the
    "pretokenization" stage.

  {.newline} - literal newline
  {.space} - literal space
  {.tab} - literal tab

  {3,4} should be interpreted correctly.
  \{3,4\} is just a literal string.


Reuse of Subpatterns:

  Proposed syntax for "subpatterns", which are like subroutines:

  {:int} -> { \d+ anon | int}
  {:int hour} -> { \d+ hour|int }
  {!integer} -> negation -- 

  {hour <:integer>} -> {hour \d+ | int}

  When these patterns are encountered, we go look up the name, and then compile
  them.  Sort of like "includes" too.


NOTES

  Instead of {/optional}, how about {/value? $name}  -- Nah, ? is too much
  punctuation, and can't be applied to choices, etc.

  then you can also do of {/choices? foo} ???  Is that a good idea?  An empty
  {/or}{/end} clause might be easier

  {:date foo}, {:date? foo}

  This is different than character classes
  {:int hour}
  {:int minute}
  {:int? minute}
  {!int minute}  -- this doesn't make sense.

Not needed yet:

  /chars:
  {/repeated value * foo}{/chars abcdef}{/end}

  :colon: - \:
  :bang: - \!
  :meta-left: \{
  :meta-right: \}
  :space:, :newline: :tab:

  Maybe I don't need colons, I can just use 
  {time}{:time}{/end}
  {time}{!digit}{/end}

  {/space} -- Or { }? 
  {/newline}
  {/tab}

  {desc :*:}  - like .*?
  : : - space

Experimental:

  {/chars a-z}
  {/chars! a-z}

  {/inc parens}
  {/dec parens}
  {/eq parens 0}
"""

__author__ = 'Andy Chu'

__all__ = ['Pattern']

import re
import sys

import execute
import parse
import tokenize  # this package, not stdlib

class Error(Exception):
  pass


class CaptureResult(object):
  """Return value of Pattern.capture.

  All attributes are public.  They're mutated by the capture process, and then
  meant to be read by the user.

  Exactly one of 'data' and 'failures' should be "true".  If the capture process
  is successful, then 'data' will contain the "JSON" dictionary.  Otherwise,
  'failures' will contain a list of failures.
  """

  def __init__(self):
    """
    Args:
      data: 
      failures: 
    """
    self.match_count = 0
    self.data = None 
    self.log_entries = []  # This can be a tree of failures, for subpatterns
    self.end = None  # End of match, unset at first

  def log(self, msg, *args):
    if args:
      msg = msg % args
    self.log_entries.append(msg)
    #log(msg)

  def MatchLog(self):
    return '\n'.join(MatchLog(self.log_entries)) + '\n'


def MatchLog(entries, indent=0):
  leading = indent * ' '
  lines = []
  for f in entries:
    if isinstance(f, list):
      lines.extend(MatchLog(f, indent=indent+4))
    else:
      lines.append(leading + f)
  return lines


class Pattern(object):

  def __init__(
      self, pattern_str, patterns={}, filters={}, actions={},
      name=None, desc=None):

    # Compile the pattern to a tree representation
    builder = parse.ProgramBuilder(
        patterns=patterns, filters=filters, actions=actions)
    tokens = tokenize.Tokenize(pattern_str)
    self._program = parse.Parse(pattern_str, tokens, builder)

    # We don't technically need to save this, but it may be useful for
    # debugging and such, and is used in the unit tests.
    self.pattern_str = pattern_str

    # Also for cosmetic purposes
    self.name = name
    self.desc = desc  # "docstring" for the pattern

  def match(self, s, pos=0, workspace=None):
    """
    Returns a dictionary of captured values, or None if there was no match.
    """
    result = CaptureResult()
    result.workspace = workspace  # Let the _Match* functions read it
    result.data, result.pos = execute._MatchValue(self._program, result, s, pos)
    return result

  def AsTree(self):
    return self._program.AsTree()

  def __str__(self):
    return '%s' % (self.name or '(anon)')
